import requests
import json
from pymongo import MongoClient
import os

client = MongoClient('mongodb://localhost:27017/')
db = client['news_db']
collection = db['news_collection']

image_dir = 'news_images'
if not os.path.exists(image_dir):
    os.makedirs(image_dir)

page_num = 1
all_responses = []

while True:
    try:
        url = f"https://news.cctv.com/2019/07/gaiban/cmsdatainterface/page/news_{page_num}.jsonp"
        params = {
            "cb": "news"
        }
        headers = {
            "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) "
                          "Chrome/135.0.0.0 Safari/537.36 Edg/135.0.0.0"
        }
        response = requests.get(url, params=params, headers=headers)
        json_data = response.content.decode().replace("news(", "").replace(")", "")
        data = json.loads(json_data)
        all_responses.append(data)
        page_num += 1
    except Exception as e:
        print(f"结束请求")
        break

# 将数据存储到 MongoDB 中
for req_one in all_responses:
    news_list = req_one["data"]["list"]
    for news in news_list:
        title = news.get('title', '')
        brief = news.get('brief', '')
        image_url = news.get('image', '')

        if image_url:
            try:
                image_response = requests.get(image_url)
                image_name = os.path.join(image_dir, os.path.basename(image_url))
                with open(image_name, 'wb') as f:
                    f.write(image_response.content)
                image_path = os.path.abspath(image_name)
            except Exception as e:
                print(f"图片下载失败: {image_url}, 错误信息: {e}")
                image_path = ''
        else:
            image_path = ''

        news_data = {
            'title': title,
            'brief': brief,
            'image_path': image_path
        }
        collection.insert_one(news_data)

# 关闭 MongoDB 连接
client.close()
print("数据存储完成")
